#!/usr/bin/env python
# encoding: utf-8
"""
Dataset.py

Created by Pablo Ortega Mesa on 2011-02-01.
Copyright (c) 2011 Toeska Research Group. All rights reserved.
"""

import sys
import os
import json
import re
import getopt
#from stemmer import *
#import MySQLdb

class Dataset():
	def __init__(self,path,output,categories=True):
		self.inputApiPath, self.inputMashupPath, self.outputApiPath, self.outputMashupPath = self.__generatePaths(path,output)
		self.byCategories = categories
		self.readDataset()
		
	def __generatePaths(self,path,output):
		api_inputPath = path+"apis/apis.json"
		if not os.path.exists(output+"apis"):
			os.mkdir(output+"apis",0777)
		api_outputPath = output+"apis/"
		mashup_inputPath = path+"mashups/"
		if not os.path.exists(output+"mashups"):
			os.mkdir(output+"mashups")
		mashup_outputPath = output+"mashups/"
		return api_inputPath,mashup_inputPath,api_outputPath,mashup_outputPath
	
	def __getApiData(self,api):
		dataApi = ''
		dataApi += api['name']+'\n'
		dataApi += api['summary'].encode("utf-8")+'\n'
		if api['summary'].lower() != api['description'].lower():
			dataApi += api['description'].encode("utf-8")+'\n'
		tags = ''
		for t in api['tags']:
			tags += t+','
			tags += '\n'
		dataApi += tags
		return dataApi
	
	def __getMashupData(self,mashup):
		dataMashup = ''
		dataMashup += mashup['name'].encode('utf-8')+'\n'
		dataMashup += mashup['summary'].encode('utf-8')+'\n'
		if mashup['summary'].lower() != mashup['description'].lower():
			dataMashup += mashup['description'].encode('utf-8')+'\n'
		tags = ''
		for t in mashup['tags']:
			tags += t+","
			tags +='\n'
		dataMashup += tags
		return dataMashup
		
	
	def readDataset(self):
		'''Reading apis dataset'''
		apis = []
		contApis = 0
		apisFile = open(self.inputApiPath,'r')
		for line in apisFile:
			if line.startswith("[{") or line.startswith("{"):
				api = '{\n'
			elif line.startswith("}]") or line.startswith("}"):
				api += '}\n'
				apis.append(json.loads(api))
				contApis+=1
			else:
				api += line
		self.apis = apis
		print str(contApis)+" APIs loaded"
		'''Reading mashups dataset'''	
		docs = os.listdir(self.inputMashupPath)
		vp = re.compile('.*(json)$')
		archives = []
		for doc in docs:
			if os.path.isfile(self.inputMashupPath+doc) and vp.search(doc):
				archives.append(doc)
		print str(len(archives))+" files readed"
		mashups = []
		contMashups = 0
		for doc in archives:
			mashupsFile = open(self.inputMashupPath+doc,'r')
			mashup = ''
			for line in mashupsFile:
				if line.startswith("[ {"):
					mashup = "{\n"
				elif line.startswith("}, {"):
					mashup += "}\n"
					mashups.append(json.loads(mashup))
					contMashups+=1
					mashup = "{\n"
				else:
					mashup += line
			mashupsFile.close()
		self.mashups = mashups
		print str(contMashups)+" MASHUPs loaded"
	
	def writeDataset(self):
		'''Write apis dataset'''
		contApis = 0
		for api in self.apis:
			filename = re.sub('[^a-zA-Z0-9_. ]','',api['name'])
			filename = re.sub(' ','_',filename)
			filename = "api_"+filename+".txt"
			fapi = open(self.outputApiPath+filename,'w')
			fapi.write(self.__getApiData(api))
			fapi.close()
			contApis += 1
		print str(contApis)+" apis writed"
		contMashups = 0
		for mashup in self.mashups:
			filename = re.sub('[^a-zA-Z0-9_ ]','',mashup['name'])
			filename = re.sub(' ','_',filename)
			filename = "mashup_"+filename+".txt"
			fmashup = open(self.outputMashupPath+filename,'w')
			fmashup.write(self.__getMashupData(mashup))
			fmashup.close()
			contMashups += 1
		print str(contMashups)+" mashups writed"
			
					
	def test(self):
		print self.inputApiPath
		print self.inputMashupPath
		print self.outputApiPath
		print self.outputMashupPath
		self.writeDataset()
		
d = Dataset('../Dataset/','../Corporas/',False)
d.test()